package weiboCrawler;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStreamWriter;
import java.io.Serializable;
import java.sql.Savepoint;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.apache.log4j.chainsaw.Main;
import org.codehaus.jackson.map.ser.impl.StaticListSerializerBase;


import com.sun.org.apache.bcel.internal.generic.NEW;
import com.sun.xml.internal.stream.buffer.stax.StreamWriterBufferCreator;

import weibo4j.Friendships;
import weibo4j.Timeline;
import weibo4j.Users;
import weibo4j.examples.oauth2.Log;
import weibo4j.model.Paging;
import weibo4j.model.Status;
import weibo4j.model.StatusWapper;
import weibo4j.model.User;
import weibo4j.model.UserWapper;
import weibo4j.model.WeiboException;
import weibo4j.org.json.JSONObject;

import weiboCrawler.Node;
import weiboCrawler.Link;

public class Crawler implements Serializable{
	
	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;
	private List<Node> nodes;
	private List<Link> links;
	
	Crawler(){
		nodes = new ArrayList<Node>();
		links = new ArrayList<Link>();
		defineInvolveKeyWords = new ArrayList<String>();
		defineInvolveKeyWords.add("的");
	}
	
	// app:test
	//private static String my_access_token = "2.00utce3C8tFiCB7a1dc3ca40ZTPehB";
	// app:test 2
	//private static String my_access_token = "2.00utce3C0Y1go1b753c889edmbpgdC";
	// app:test3
	//private static String my_access_token = "2.00utce3C_pV48C91a1e6363bqlfGMB";
	// app: test 4
	private static String my_access_token = "2.00utce3CBb7eWE562e69a134xM9DKD";
	
		// Max posts crawled per person
	private static int MAX_POST_PER_PERSON = 300;
		// Max friend number in crawling the web
	private static int MAX_FRIEND_NUM = 50;
		// Friendship search branching probability
	private static float branchProb = 0.2f;
	
		// friend search direction:
		// 1: crawl along friend
		// -1: crawl along follower
	private static int searchDirection = 1;
	
	private static List<String> defineInvolveKeyWords;
	
	/*
	 * 获取个人文集
	 * num为微博条数
	 * 文集限于某个时间段之前，返回微博数目可能小于给定条数
	 */
	private static List<String> getCorpusByName(String screenName, int maxPostNum){

		List<String> ans = new ArrayList<String>();
		
		Timeline tm = new Timeline();
		tm.client.setToken(my_access_token);

		long since_id = 3441878861916208l;
		
		try {
			boolean out_of_range = false;
			StatusWapper statusWapper = tm.getUserTimelineByName(screenName, new Paging(1, 200), 0, 0);
			// first page
			for (Status status : statusWapper.getStatuses()) {
				if (Long.parseLong(status.getId()) < since_id || ans.size() >= maxPostNum ){
					out_of_range = true;
					break;
				}
				//bw.write(Long.parseLong(status.getId()) + ":" + status.getCreatedAt().toString() + status.getText()+"\r\n");
				ans.add(status.getText());
			}
			// following pages
			for ( int i = 1; i < (statusWapper.getTotalNumber() - 1) / 200 + 1; i ++ ){
				if ( out_of_range ) break;
				statusWapper = tm.getUserTimelineByName(screenName, new Paging(i, 200), 0, 0);
				for (Status status : statusWapper.getStatuses()) {
					if (Long.parseLong(status.getId()) < since_id || ans.size() >= maxPostNum ){
						out_of_range = true;
						break;
					}
					//bw.write(Long.parseLong(status.getId()) + ":" + status.getCreatedAt().toString() + status.getText()+"\r\n");
					ans.add(status.getText());
				}
			}			
			
		} catch (WeiboException e) {
			//e.printStackTrace();
		}
		
		return ans;
	}
	
	
	/*
	 * Get the "background corpus" from the nodes expanded by originScreenName
	 */
	private List<String> getCorpusBase(String originScreenName){
		
		List<String> ans = new ArrayList<String>();
		initNode(originScreenName, 10, 300);
		for (Node node : nodes){
			ans.addAll(getCorpusByName(node.screen_name, 100));
		}
		
		return ans;
	}
	
	/*
	 * 获取用户首次“相关时间
	 * maxPostNum: 最大搜索条数
	 */
	private static String getFirstInvolvedDate(String screenName, List<String> wordList, int maxPostNum ) throws WeiboException{
				
		Timeline tm = new Timeline();
		tm.client.setToken(my_access_token);

		long since_id = 3503963962251544l;
		
		Date date = new Date();
		long status_id = 0;
		
		try {
			boolean out_of_range = false;
			int cnt= 0;
			StatusWapper statusWapper = tm.getUserTimelineByName(screenName, new Paging(1, 50), 0, 0);
			// first page
			for (Status status : statusWapper.getStatuses()) {
				if (Long.parseLong(status.getId()) < since_id || cnt >= maxPostNum ){
					out_of_range = true;
					break;
				}
				//bw.write(Long.parseLong(status.getId()) + ":" + status.getCreatedAt().toString() + status.getText()+"\r\n");
				for ( int i = 0; i < wordList.size(); i ++ )
					if (status.getText().contains(wordList.get(i)))
					{
						date = status.getCreatedAt();
						status_id = Long.parseLong(status.getId());
					}
				cnt ++;
			}
			// following pages
			for ( int i = 1; i < (statusWapper.getTotalNumber() - 1) / 50 + 1; i ++ ){
				if ( out_of_range ) break;
				statusWapper = tm.getUserTimelineByName(screenName, new Paging(i, 50), 0, 0);
				for (Status status : statusWapper.getStatuses()) {
					if (Long.parseLong(status.getId()) < since_id || cnt >= maxPostNum ){
						out_of_range = true;
						break;
					}
					//bw.write(Long.parseLong(status.getId()) + ":" + status.getCreatedAt().toString() + status.getText()+"\r\n");
					for ( int j = 0; j < wordList.size(); j ++ )
						if (status.getText().contains(wordList.get(j)))
						{
							date = status.getCreatedAt();
							status_id = Long.parseLong(status.getId());
						}
					cnt ++;
				}
			}			
			
		} catch (WeiboException e) {
			throw e;
		}

			System.out.println(status_id);
			SimpleDateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss z yyyy", Locale.ENGLISH);
			return formatter.format(date);
		
	}
	
	/*
	 * Initialize nodes by crawling from the origin
	 */
	public void initNode(String originName, int depth, int maxUserCnt){
		/*
		 * =======================
		 * 
		 *  TODO: implement this
		 *   
		 * =======================
		 */
		nodes = new ArrayList<Node>();

		// Begin with root user;
		nodes.add(new Node(originName));
		
		Friendships fm = new Friendships();
		fm.client.setToken(my_access_token);
		
		try {
			int i = 0;
			int nextGenStart = 0;
			int curDepth = 0;
			while (i < getNodes().size() && nodes.size() < maxUserCnt && curDepth < depth){
				if ( i >= nextGenStart){
					nextGenStart = getNodes().size();
					curDepth ++;
				}
				UserWapper users = fm.getFriendsByScreenName(getNodes().get(i).screen_name);
				for(User u : users.getUsers()){
					if ( nodes.size() > maxUserCnt ) break;
					// branching probability
					if ( Math.random() > branchProb ){
						System.out.println("cut!");
						continue;
					}
					
					if (u.getVerifiedType()==0)
					{
						int linkTarget = -1;	// default: the newly inserted node
						// check for existing
						for(int j = 0; j < nodes.size(); j ++){
							Node node = nodes.get(j);
							if (node.screen_name == u.getScreenName()){
								linkTarget = j;	
							}
						}
						if ( linkTarget == -1 ){
							linkTarget = nodes.size();
							// Insert Node
							Node node = new Node(u.getScreenName(), u.getLocation());
							node.location = u.getLocation();
							nodes.add(node);
						}
						// Insert Link
						links.add(new Link(i, linkTarget, 1));
					}
				}
				i ++;
			}
		} catch (WeiboException e) {
			e.printStackTrace();
		}
		
	}
	
	/*
	 * Initialize the links
	 */
	public void initLink(){
		
		links = new ArrayList<Link>();
		
		for (int i = 0; i < getNodes().size(); i++) 
			for (int j = i + 1; j < getNodes().size(); j ++){
				links.add(new Link(i, j, getDist(getNodes().get(i), getNodes().get(j))));
			}
		
	}
	
	/*
	 * cache: 
	 */
	public void initTime(int useCache){
		if (useCache == 1)
			loadState();
		
		for ( int i = 0; i < nodes.size(); i ++ ){
			Node node = nodes.get(i);
		
			if (node.firstInvolveDate == null ){
				
				try{
					node.firstInvolveDate = getFirstInvolvedDate(node.screen_name, defineInvolveKeyWords, 300);
					nodes.set(i, node);
				}
				catch(Exception exception){
					break;
				}
			}
		}
		saveState();
		
		/*
		 * count num
		 */
		int num = 0;
		for( Node node : nodes){
			if (node.firstInvolveDate != null )
				num ++;
		}
		System.out.println("time number" + num);
	}
	
	/*
	 * 通过TF-IDF方法获取关键词
	 */
	public List<String> getKeyWords(List<String> corpus){
		return null;
	}
	
	/*
	 * Get distance of any pair of{ users
	 */
	private float getDist(Node usr1, Node usr2){
		/*
		 * =======================
		 * 
		 *  TODO: implement this
		 *   
		 * =======================
		 */
		return 3.0f;
	}
	
	public void saveJSON() throws IOException{
		
		Map<String, Object> map = new HashMap<String, Object>();
		
		// node
		List<Map<String, Object>> nodeList = new ArrayList<Map<String, Object>>();
		Map<String, Object> nodeMap;
		
			// Insert a node
		for (int i = 0; i < getNodes().size(); i ++){
			nodeMap = new HashMap<String, Object>();
			nodeMap.put("name", getNodes().get(i).screen_name);
			nodeMap.put("group", new Integer(1));
			nodeMap.put("location", nodes.get(i).location);
			
			String dateStr = nodes.get(i).firstInvolveDate;
			if (dateStr == null ){
				SimpleDateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss z yyyy", Locale.ENGLISH);
				dateStr = formatter.format(new Date());
			}
			nodeMap.put("date", dateStr);
			
			nodeList.add(nodeMap);
		}
		
		map.put("nodes", nodeList);
		
		// link
		List<Map<String, Integer>> linkList = new ArrayList<Map<String,Integer>>();
		Map<String, Integer> linkMap;
		
			// Insert a link
		for ( int i = 0; i < links.size(); i ++){
			linkMap = new HashMap<String, Integer>();
			linkMap.put("source", links.get(i).n1);
			linkMap.put("target", links.get(i).n2);
			linkMap.put("value", 1);
			linkList.add(linkMap);
			/*
			 * =======================
			 * 
			 *  TODO: add distance to JSON
			 *   
			 * =======================
			 */
		}
		
		map.put("links", linkList);
		
		// Write out
		JSONObject json = new JSONObject(map);
		FileOutputStream stream = new FileOutputStream("a3.json");
		StringBuffer sb = new StringBuffer();
		sb.append(json.toString());
		stream.write(json.toString().getBytes());
		stream.close();
		
	}
	
	public  void saveState(){
		try{   
            FileOutputStream fs = new FileOutputStream("nodes.ser");   
            ObjectOutputStream os =  new ObjectOutputStream(fs);   
            os.writeObject(nodes);   
            os.close();   
            fs = new FileOutputStream("links.ser");
            os = new ObjectOutputStream(fs);
            os.writeObject(links);
            os.close();
        }catch(Exception ex){   
            ex.printStackTrace();   
        }
	}
	
	@SuppressWarnings("unchecked")
	public void loadState(){
		try {
			   FileInputStream fs = new FileInputStream("nodes.ser");
			   ObjectInputStream ois = new ObjectInputStream(fs);
			   nodes = (ArrayList<Node>) ois.readObject();
			   ois.close();
			   fs = new FileInputStream("links.ser");
			   ois = new ObjectInputStream(fs);
			   links = (ArrayList<Link>) ois.readObject();
			   ois.close();
			  } catch (Exception ex) {
			   ex.printStackTrace();
			  }
			 
	}

	public static void main(String[] args) throws IOException, WeiboException {

		FileOutputStream fos = new FileOutputStream("test.out.txt");
		OutputStreamWriter osw = new OutputStreamWriter(fos);
		BufferedWriter bw = new BufferedWriter(osw);
		
		/* 
		 * Example: get first involve date.
		 * keyWordList is the list of keywords that determins a "involve".
		 */
	
		List<String> keyWordList = new ArrayList<String>();
		keyWordList.add("爆表");
		System.out.println(getFirstInvolvedDate("The_Mechanic", keyWordList, 1000));
		
		

	}

	public List<Node> getNodes() {
		return nodes;
	}

	public void setNodes(List<Node> nodes) {
		nodes = nodes;
	}

	
	
	
}
